import json
from followUserSpider.items import FollowUserItem
from followUserSpider.items import FinishUserItem
from scrapy_redis.spiders import RedisCrawlSpider
from scrapy.http import Request
from followUserSpider.settings import INIT_QUEUE,DONE_QUEQE,EXECUTE_QUEUE
from followUserSpider.redisUtils import OPRedis
from scrapy import log

class followSpider(RedisCrawlSpider):

    name = "follow"

    redis_key = INIT_QUEUE

    # 用户关注
    follow_url = 'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_{uid}&page={page}'

    opr = OPRedis()

    def make_requests_from_url(self, url):
        #判断id是否已经在处理或者处理完成
        if (self.opr.sismember(EXECUTE_QUEUE,url)|self.opr.sismember(DONE_QUEQE,url)):
            return
        else:
            #把id加入任务处理中的队列
            self.opr.sadd(EXECUTE_QUEUE,url)
            return Request(self.follow_url.format(uid=url, page=1),dont_filter=True ,
                           callback=self.parse,meta={'page': 1, 'uid': url})

    def parse(self, response):
        page = response.meta.get('page')
        uid = response.meta.get('uid')
        followUserids = []
        try:
            jsonData = json.loads(response.text)
        except Exception as e:
            log.msg("请求报错", level=log.INFO)
            log.msg(response.text, level=log.INFO)
            return
        if (jsonData.get('ok') == 1):
            data = jsonData.get('data')
            cards = data.get('cards')
            for card in cards:
                card_group = card.get('card_group')
                for j in range(len(card_group)):
                    group = card_group[j]
                    card_type = group.get('card_type')
                    if card_type==10:
                        user = card_group[j].get('user')
                        id = user.get('id')
                        followUserids.append(id)
            followUserItem = FollowUserItem()
            followUserItem['userId'] = uid
            followUserItem['followUserids'] = followUserids
            yield followUserItem
            page+=1
            yield Request(self.follow_url.format(uid=uid, page=page), callback=self.parse, meta={'uid': uid,'page':page},dont_filter=True)
        else:
            #单个用户关注列表爬取结束
            finishUserItem = FinishUserItem()
            finishUserItem['userId'] = uid
            yield finishUserItem